{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 2.1.0\n",
      "\n",
      "data loading ...\n",
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n",
      "`ranking_task` initialized with metrics [normalized_discounted_cumulative_gain@3(0.0), normalized_discounted_cumulative_gain@5(0.0), mean_average_precision(0.0)]\n",
      "loading embedding ...\n",
      "embedding loaded as `glove_embedding`\n"
     ]
    }
   ],
   "source": [
    "%run ./tutorials/wikiqa/init.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.optimizers import Adam\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "import matchzoo as mz\n",
    "from matchzoo.contrib.models.esim import ESIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_filtered_data(preprocessor, data_type):\n",
    "    assert ( data_type in ['train', 'dev', 'test'])\n",
    "    data_pack = mz.datasets.wiki_qa.load_data(data_type, task='ranking')\n",
    "\n",
    "    if data_type == 'train':\n",
    "        X, Y = preprocessor.fit_transform(data_pack).unpack()\n",
    "    else:\n",
    "        X, Y = preprocessor.transform(data_pack).unpack()\n",
    "\n",
    "    new_idx = []\n",
    "    for i in range(Y.shape[0]):\n",
    "        if X[\"length_left\"][i] == 0 or X[\"length_right\"][i] == 0:\n",
    "            continue\n",
    "        new_idx.append(i)\n",
    "    new_idx = np.array(new_idx)\n",
    "    print(\"Removed empty data. Found \", (Y.shape[0] - new_idx.shape[0]))\n",
    "\n",
    "    for k in X.keys():\n",
    "        X[k] = X[k][new_idx]\n",
    "    Y = Y[new_idx]\n",
    "\n",
    "    pos_idx = (Y == 1)[:, 0]\n",
    "    pos_qid = X[\"id_left\"][pos_idx]\n",
    "    keep_idx_bool = np.array([ qid in pos_qid for qid in X[\"id_left\"]])\n",
    "    keep_idx = np.arange(keep_idx_bool.shape[0])\n",
    "    keep_idx = keep_idx[keep_idx_bool]\n",
    "    print(\"Removed questions with no pos label. Found \", (keep_idx_bool == 0).sum())\n",
    "\n",
    "    print(\"shuffling...\")\n",
    "    np.random.shuffle(keep_idx)\n",
    "    for k in X.keys():\n",
    "        X[k] = X[k][keep_idx]\n",
    "    Y = Y[keep_idx]\n",
    "\n",
    "    return X, Y, preprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fixed_length_left = 10\n",
    "fixed_length_right = 40\n",
    "batch_size = 32\n",
    "epochs = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 10798.93it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:02<00:00, 8019.65it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 1415354.12it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 226166.63it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 233892.08it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 782897.32it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 1175423.27it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 358408/358408 [00:00<00:00, 4845654.07it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 15108.05it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:02<00:00, 8129.15it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 222548.25it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 324738.11it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 122413.67it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 821484.73it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 1319786.92it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 200871.36it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 180842.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed empty data. Found  91\n",
      "Removed questions with no pos label. Found  11642\n",
      "shuffling...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 296/296 [00:00<00:00, 15853.43it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2708/2708 [00:00<00:00, 8318.22it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2708/2708 [00:00<00:00, 232964.32it/s]\n",
      "Processing text_left with transform: 100%|██████████| 296/296 [00:00<00:00, 200892.23it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2708/2708 [00:00<00:00, 231808.96it/s]\n",
      "Processing length_left with len: 100%|██████████| 296/296 [00:00<00:00, 562279.88it/s]\n",
      "Processing length_right with len: 100%|██████████| 2708/2708 [00:00<00:00, 1159470.73it/s]\n",
      "Processing text_left with transform: 100%|██████████| 296/296 [00:00<00:00, 183357.55it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2708/2708 [00:00<00:00, 178815.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed empty data. Found  8\n",
      "Removed questions with no pos label. Found  1595\n",
      "shuffling...\n"
     ]
    }
   ],
   "source": [
    "# prepare data\n",
    "preprocessor = mz.preprocessors.BasicPreprocessor(fixed_length_left=fixed_length_left,\n",
    "                                                  fixed_length_right=fixed_length_right,\n",
    "                                                  remove_stop_words=False,\n",
    "                                                  filter_low_freq=10)\n",
    "\n",
    "train_X, train_Y, preprocessor = load_filtered_data(preprocessor, 'train')\n",
    "val_X, val_Y, _ = load_filtered_data(preprocessor, 'dev')\n",
    "pred_X, pred_Y = val_X, val_Y\n",
    "# pred_X, pred_Y, _ = load_filtered_data(preprocessor, 'test') # no prediction label for quora dataset\n",
    "\n",
    "embedding_matrix = glove_embedding.build_matrix(preprocessor.context['vocab_unit'].state['term_index'], initializer=lambda: 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 10)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 40)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           multiple             1930500     text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             multiple             0           embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "                                                                 dense_1[0][0]                    \n",
      "                                                                 dense_1[1][0]                    \n",
      "                                                                 dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               multiple             0           text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) multiple             1442400     dropout_1[0][0]                  \n",
      "                                                                 dropout_1[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 10, 1)        0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 40, 1)        0           lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 10, 600)      0           bidirectional_1[0][0]            \n",
      "                                                                 lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 40, 600)      0           bidirectional_1[1][0]            \n",
      "                                                                 lambda_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, 10, 1)        0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, 1, 40)        0           lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, 10, 40)       0           multiply_1[0][0]                 \n",
      "                                                                 multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 10, 40)       0           lambda_4[0][0]                   \n",
      "                                                                 lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "permute_1 (Permute)             (None, 40, 10)       0           dot_1[0][0]                      \n",
      "                                                                 multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "atten_mask (Lambda)             multiple             0           dot_1[0][0]                      \n",
      "                                                                 multiply_3[0][0]                 \n",
      "                                                                 permute_1[0][0]                  \n",
      "                                                                 permute_1[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "softmax_1 (Softmax)             multiple             0           atten_mask[0][0]                 \n",
      "                                                                 atten_mask[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_2 (Dot)                     (None, 10, 600)      0           softmax_1[0][0]                  \n",
      "                                                                 multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_3 (Dot)                     (None, 40, 600)      0           softmax_1[1][0]                  \n",
      "                                                                 multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "subtract_1 (Subtract)           (None, 10, 600)      0           multiply_1[0][0]                 \n",
      "                                                                 dot_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "multiply_4 (Multiply)           (None, 10, 600)      0           multiply_1[0][0]                 \n",
      "                                                                 dot_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "subtract_2 (Subtract)           (None, 40, 600)      0           multiply_2[0][0]                 \n",
      "                                                                 dot_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "multiply_5 (Multiply)           (None, 40, 600)      0           multiply_2[0][0]                 \n",
      "                                                                 dot_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 10, 2400)     0           multiply_1[0][0]                 \n",
      "                                                                 dot_2[0][0]                      \n",
      "                                                                 subtract_1[0][0]                 \n",
      "                                                                 multiply_4[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 40, 2400)     0           multiply_2[0][0]                 \n",
      "                                                                 dot_3[0][0]                      \n",
      "                                                                 subtract_2[0][0]                 \n",
      "                                                                 multiply_5[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 multiple             720300      concatenate_1[0][0]              \n",
      "                                                                 concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) multiple             1442400     dropout_1[2][0]                  \n",
      "                                                                 dropout_1[3][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, 10, 1)        0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_8 (Lambda)               (None, 10, 1)        0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_10 (Lambda)              (None, 40, 1)        0           lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_12 (Lambda)              (None, 40, 1)        0           lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_6 (Multiply)           (None, 10, 600)      0           bidirectional_2[0][0]            \n",
      "                                                                 lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_7 (Multiply)           (None, 10, 600)      0           bidirectional_2[0][0]            \n",
      "                                                                 lambda_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_8 (Multiply)           (None, 40, 600)      0           bidirectional_2[1][0]            \n",
      "                                                                 lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_9 (Multiply)           (None, 40, 600)      0           bidirectional_2[1][0]            \n",
      "                                                                 lambda_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, 600)          0           multiply_6[0][0]                 \n",
      "                                                                 lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_9 (Lambda)               (None, 600)          0           multiply_7[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_11 (Lambda)              (None, 600)          0           multiply_8[0][0]                 \n",
      "                                                                 lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_13 (Lambda)              (None, 600)          0           multiply_9[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 1200)         0           lambda_7[0][0]                   \n",
      "                                                                 lambda_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 1200)         0           lambda_11[0][0]                  \n",
      "                                                                 lambda_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 2400)         0           concatenate_3[0][0]              \n",
      "                                                                 concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 300)          720300      concatenate_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1)            301         dropout_1[4][0]                  \n",
      "==================================================================================================\n",
      "Total params: 6,256,201\n",
      "Trainable params: 4,325,701\n",
      "Non-trainable params: 1,930,500\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = ESIM()\n",
    "model.params['task'] = mz.tasks.Ranking()\n",
    "model.params['mask_value'] = 0\n",
    "model.params['input_shapes'] = [[fixed_length_left, ],\n",
    "                                [fixed_length_right, ]]\n",
    "model.params['lstm_dim'] = 300\n",
    "model.params['embedding_input_dim'] = preprocessor.context['vocab_size']\n",
    "model.params['embedding_output_dim'] = 300\n",
    "model.params['embedding_trainable'] = False\n",
    "model.params['dropout_rate'] = 0.5\n",
    "\n",
    "model.params['mlp_num_units'] = 300\n",
    "model.params['mlp_num_layers'] = 0\n",
    "model.params['mlp_num_fan_out'] = 300\n",
    "model.params['mlp_activation_func'] = 'tanh'\n",
    "model.params['optimizer'] = Adam(lr=4e-4)\n",
    "\n",
    "model.guess_and_fill_missing_params()\n",
    "model.build()\n",
    "\n",
    "model.compile()\n",
    "model.backend.summary() # not visualize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8627 samples, validate on 1130 samples\n",
      "Epoch 1/5\n",
      "8627/8627 [==============================] - 48s 6ms/step - loss: 0.1073 - val_loss: 0.0984\n",
      "Validation: mean_average_precision(0.0): 0.6222655981584554\n",
      "Epoch 2/5\n",
      "8627/8627 [==============================] - 44s 5ms/step - loss: 0.0994 - val_loss: 0.0974\n",
      "Validation: mean_average_precision(0.0): 0.640342571890191\n",
      "Epoch 3/5\n",
      "8627/8627 [==============================] - 44s 5ms/step - loss: 0.0944 - val_loss: 0.0981\n",
      "Validation: mean_average_precision(0.0): 0.633281742507933\n",
      "Epoch 4/5\n",
      "8627/8627 [==============================] - 44s 5ms/step - loss: 0.0915 - val_loss: 0.0898\n",
      "Validation: mean_average_precision(0.0): 0.6479046351993808\n",
      "Epoch 5/5\n",
      "8627/8627 [==============================] - 44s 5ms/step - loss: 0.0893 - val_loss: 0.0931\n",
      "Validation: mean_average_precision(0.0): 0.6506805763854636\n"
     ]
    }
   ],
   "source": [
    "# run as classification task\n",
    "model.load_embedding_matrix(embedding_matrix)\n",
    "evaluate = mz.callbacks.EvaluateAllMetrics(model,\n",
    "                                           x=pred_X,\n",
    "                                           y=pred_Y,\n",
    "                                           once_every=1,\n",
    "                                           batch_size=len(pred_Y))\n",
    "\n",
    "history = model.fit(x = [train_X['text_left'],\n",
    "                         train_X['text_right']],\n",
    "                    y = train_Y,\n",
    "                    validation_data = (val_X, val_Y),\n",
    "                    batch_size = batch_size,\n",
    "                    epochs = epochs,\n",
    "                    callbacks=[evaluate]\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text_left (InputLayer)          (None, 10)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "text_right (InputLayer)         (None, 40)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           multiple             1930500     text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             multiple             0           embedding[0][0]                  \n",
      "                                                                 embedding[1][0]                  \n",
      "                                                                 dense_1[0][0]                    \n",
      "                                                                 dense_1[1][0]                    \n",
      "                                                                 dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               multiple             0           text_left[0][0]                  \n",
      "                                                                 text_right[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) multiple             1442400     dropout_1[0][0]                  \n",
      "                                                                 dropout_1[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 10, 1)        0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 40, 1)        0           lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 10, 600)      0           bidirectional_1[0][0]            \n",
      "                                                                 lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 40, 600)      0           bidirectional_1[1][0]            \n",
      "                                                                 lambda_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, 10, 1)        0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, 1, 40)        0           lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, 10, 40)       0           multiply_1[0][0]                 \n",
      "                                                                 multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 10, 40)       0           lambda_4[0][0]                   \n",
      "                                                                 lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "permute_1 (Permute)             (None, 40, 10)       0           dot_1[0][0]                      \n",
      "                                                                 multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "atten_mask (Lambda)             multiple             0           dot_1[0][0]                      \n",
      "                                                                 multiply_3[0][0]                 \n",
      "                                                                 permute_1[0][0]                  \n",
      "                                                                 permute_1[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "softmax_1 (Softmax)             multiple             0           atten_mask[0][0]                 \n",
      "                                                                 atten_mask[1][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_2 (Dot)                     (None, 10, 600)      0           softmax_1[0][0]                  \n",
      "                                                                 multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dot_3 (Dot)                     (None, 40, 600)      0           softmax_1[1][0]                  \n",
      "                                                                 multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "subtract_1 (Subtract)           (None, 10, 600)      0           multiply_1[0][0]                 \n",
      "                                                                 dot_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "multiply_4 (Multiply)           (None, 10, 600)      0           multiply_1[0][0]                 \n",
      "                                                                 dot_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "subtract_2 (Subtract)           (None, 40, 600)      0           multiply_2[0][0]                 \n",
      "                                                                 dot_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "multiply_5 (Multiply)           (None, 40, 600)      0           multiply_2[0][0]                 \n",
      "                                                                 dot_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 10, 2400)     0           multiply_1[0][0]                 \n",
      "                                                                 dot_2[0][0]                      \n",
      "                                                                 subtract_1[0][0]                 \n",
      "                                                                 multiply_4[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 40, 2400)     0           multiply_2[0][0]                 \n",
      "                                                                 dot_3[0][0]                      \n",
      "                                                                 subtract_2[0][0]                 \n",
      "                                                                 multiply_5[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 multiple             720300      concatenate_1[0][0]              \n",
      "                                                                 concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) multiple             1442400     dropout_1[2][0]                  \n",
      "                                                                 dropout_1[3][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, 10, 1)        0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_8 (Lambda)               (None, 10, 1)        0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_10 (Lambda)              (None, 40, 1)        0           lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_12 (Lambda)              (None, 40, 1)        0           lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_6 (Multiply)           (None, 10, 600)      0           bidirectional_2[0][0]            \n",
      "                                                                 lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_7 (Multiply)           (None, 10, 600)      0           bidirectional_2[0][0]            \n",
      "                                                                 lambda_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_8 (Multiply)           (None, 40, 600)      0           bidirectional_2[1][0]            \n",
      "                                                                 lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_9 (Multiply)           (None, 40, 600)      0           bidirectional_2[1][0]            \n",
      "                                                                 lambda_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, 600)          0           multiply_6[0][0]                 \n",
      "                                                                 lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_9 (Lambda)               (None, 600)          0           multiply_7[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_11 (Lambda)              (None, 600)          0           multiply_8[0][0]                 \n",
      "                                                                 lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_13 (Lambda)              (None, 600)          0           multiply_9[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 1200)         0           lambda_7[0][0]                   \n",
      "                                                                 lambda_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 1200)         0           lambda_11[0][0]                  \n",
      "                                                                 lambda_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 2400)         0           concatenate_3[0][0]              \n",
      "                                                                 concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 300)          720300      concatenate_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 2)            602         dropout_1[4][0]                  \n",
      "==================================================================================================\n",
      "Total params: 6,256,502\n",
      "Trainable params: 4,326,002\n",
      "Non-trainable params: 1,930,500\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# run as classification task\n",
    "classification_task = mz.tasks.Classification(num_classes=2)\n",
    "classification_task.metrics = 'acc'\n",
    "\n",
    "model = ESIM()\n",
    "model.params['task'] = classification_task\n",
    "model.params['mask_value'] = 0\n",
    "model.params['input_shapes'] = [[fixed_length_left, ],\n",
    "                                [fixed_length_right, ]]\n",
    "model.params['lstm_dim'] = 300\n",
    "model.params['embedding_input_dim'] = preprocessor.context['vocab_size']\n",
    "model.params['embedding_output_dim'] = 300\n",
    "model.params['embedding_trainable'] = False\n",
    "model.params['dropout_rate'] = 0.5\n",
    "\n",
    "model.params['mlp_num_units'] = 300\n",
    "model.params['mlp_num_layers'] = 0\n",
    "model.params['mlp_num_fan_out'] = 300\n",
    "model.params['mlp_activation_func'] = 'tanh'\n",
    "model.params['optimizer'] = Adam(lr=4e-4)\n",
    "\n",
    "model.guess_and_fill_missing_params()\n",
    "model.build()\n",
    "\n",
    "model.compile()\n",
    "model.backend.summary() # not visualize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8627 samples, validate on 1130 samples\n",
      "Epoch 1/5\n",
      "8627/8627 [==============================] - 48s 6ms/step - loss: 0.3607 - val_loss: 0.3330\n",
      "Validation: categorical_accuracy: 1.0\n",
      "Epoch 2/5\n",
      "8627/8627 [==============================] - 43s 5ms/step - loss: 0.3273 - val_loss: 0.3490\n",
      "Validation: categorical_accuracy: 0.9451327323913574\n",
      "Epoch 3/5\n",
      "8627/8627 [==============================] - 44s 5ms/step - loss: 0.3096 - val_loss: 0.3498\n",
      "Validation: categorical_accuracy: 0.9938052892684937\n",
      "Epoch 4/5\n",
      "8627/8627 [==============================] - 44s 5ms/step - loss: 0.2970 - val_loss: 0.3170\n",
      "Validation: categorical_accuracy: 0.969911515712738\n",
      "Epoch 5/5\n",
      "8627/8627 [==============================] - 44s 5ms/step - loss: 0.2787 - val_loss: 0.3543\n",
      "Validation: categorical_accuracy: 0.8778761029243469\n"
     ]
    }
   ],
   "source": [
    "evaluate = mz.callbacks.EvaluateAllMetrics(model,\n",
    "                                           x=pred_X,\n",
    "                                           y=pred_Y,\n",
    "                                           once_every=1,\n",
    "                                           batch_size=len(pred_Y))\n",
    "\n",
    "train_Y = to_categorical(train_Y)\n",
    "val_Y = to_categorical(val_Y)\n",
    "\n",
    "model.load_embedding_matrix(embedding_matrix)\n",
    "history = model.fit(x = [train_X['text_left'],\n",
    "                         train_X['text_right']],\n",
    "                    y = train_Y,\n",
    "                    validation_data = (val_X, val_Y),\n",
    "                    batch_size = batch_size,\n",
    "                    epochs = epochs,\n",
    "                    callbacks=[evaluate]\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'categorical_accuracy': 0.8920354}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(val_X, val_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mz_play",
   "language": "python",
   "name": "mz_play"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
